package spider

import (
	"fmt"
	"log"
	"time"

	"github.com/gocolly/colly"
	"github.com/gocolly/colly/debug"
	"github.com/gocolly/colly/extensions"
)

var imgPath = "/data01/images/20210818/"

func NewCollector() *colly.Collector {
	c := colly.NewCollector(
		colly.Debugger(&debug.LogDebugger{}),
		colly.Async(false),
	)
	//配置反爬策略(设置ua和refer扩展)
	extensions.RandomUserAgent(c)
	extensions.Referer(c)

	c.Limit(&colly.LimitRule{
		// Delay:       5 * time.Second,
		DomainGlob:  "haha.sogou.com/",
		Parallelism: 2,
		RandomDelay: 10 * time.Second,
	})

	c.OnRequest(func(r *colly.Request) {
		// fmt.Println("Visiting", r.URL)
	})

	c.OnError(func(_ *colly.Response, err error) {
		log.Println("Something went wrong:", err)
	})

	c.OnResponse(func(r *colly.Response) {
		// fmt.Println("Visited", r.Request.URL)
	})

	return c
}

func Spid() {
	c := colly.NewCollector()

	c.OnRequest(func(r *colly.Request) {
		fmt.Println("Visiting", r.URL)
	})

	c.OnError(func(_ *colly.Response, err error) {
		log.Println("Something went wrong:", err)
	})

	c.OnResponse(func(r *colly.Response) {
		fmt.Println("Visited", r.Request.URL)
	})

	c.OnHTML("div[class='title']", func(e *colly.HTMLElement) {
		e.ForEach("h2", func(i int, item *colly.HTMLElement) {
			title := item.ChildText("a")
			fmt.Println("title", title)
		})
	})

	c.Visit("https://www.csdn.net/nav/other?spm=1000.2115.3001.4125")
}
